<?
$CACHE='pycode.cache';
$COUNT_THRESHOLD=4;

require_once('unicode.php');

function generate_pinyin_to_utf8_table($file)
{
  printf("Loading pinyin_to_utf8 table from %s...",$file);
  flush();
  $fp = fopen($file,'r');
  
  $entries = array();
  while($line = fgets($fp)) {
    $line = rtrim($line,"\n");
    $parts = split(sprintf('[ %s]',chr(9)),$line);
    
    $txt = ereg_replace('[0-9]','',$parts[0]);
    
    if(key_exists($txt,$entries)) {
      $entry = $entries[$txt];
    }
    else {
      $entry = array();
    }
    
    for($i=1;$i<count($parts);$i++) {
      $parts[$i] = ereg_replace('[0-9]','',$parts[$i]);
      if(strlen($parts[$i]) > 0) $entry[] = $parts[$i];
    }
    $entries[$txt] = $entry;
  }
  fclose($fp);
  printf("Done (%d of pinyin loaded)\n",count($entries));
  return $entries;
}
function generate_utf8_to_pinyin_table($pinyin_to_utf8)
{
  printf("Generating utf8_to_pinyin table...");
  flush();
  $entries = array();
  foreach($pinyin_to_utf8 as $pinyin => $utf8_array) {
    foreach($utf8_array as $utf8) {
      if(key_exists($utf8,$entries)) $entry = $entries[$utf8];
      else $entry = array();
      $entry[] = $pinyin;
      $entries[$utf8] = $entry;
    }
  }
  printf("Done (%d of utf8 char generated)\n",count($entries));
  return $entries;
}
function word_counts_filter($ar)
{
  global $COUNT_THRESHOLD;

  $ret = array();
  foreach($ar as $key => $value) {
    if($value > $COUNT_THRESHOLD || utf8strlen($key) == 1) $ret[$key] = $value; 
  }
  return $ret;
}
function generate_word_counts($file,$utf8_to_pinyin,$maxlen)
{
  printf("Generating word relations from %s (maxlen=%d)...",$file,$maxlen);
  flush();

  $stringbuf = array();
  $stringbuf_a = array();

  for($i=1;$i<=$maxlen;$i++) $stringbuf_a[$i] = false;

  $fp = fopen($file,'rb');

  $counts = array();
  while(!feof($fp)) {
    $ch = fgetc($fp);
    if($ch === false) break;
    $str = $ch;
    $len = utf8len($str);
    while(strlen($str) < $len) {
      $ch = fgetc($fp);
      if($ch === false) die("Incomplete utf8 character.\n");
      $str .= $ch;
    }
    if(!key_exists($str,$utf8_to_pinyin)) continue; // skip unknown characters
    
    for($i=1;$i<=$maxlen;$i++) {
      if($stringbuf_a[$i]) {
	$tmp = $stringbuf[$i];
	$counts[$tmp]++;
      }

      $stringbuf[$i] .= $str;
      $tmp = $stringbuf[$i];

      if(!$stringbuf_a[$i] && utf8strlen($tmp) == $i) {
	$stringbuf_a[$i] = true;
	continue;
      }
      if($stringbuf_a[$i]) $stringbuf[$i] = substr($tmp,utf8len($tmp[0]));
    }
  }
  fclose($fp);
  $counts = word_counts_filter($counts);
  printf("Done (%d relations generated)\n",count($counts));
  return $counts;
}
function sort_compare1($a,$b)
{
  $ca = $a['count'];
  $cb = $b['count'];
  if($ca < $cb) return 1;
  else if($ca > $cb) return -1;
  else {
    $ua = readutf8($a['ch']);
    $ub = readutf8($b['ch']);
    if($ua < $ub) return -1;
    else if($ua > $ub) return 1;
    else return 0;
  }
}
function sort_compare2($a,$b)
{
  $ca = $a['count'];
  $cb = $b['count'];
  if($ca < $cb) return 1;
  else if($ca > $cb) return -1;
  else return 0;
}
function generate_final_table($pinyin_to_utf8,$utf8_to_pinyin,$word_counts)
{
  printf("Generating final table...");
  flush();
  $table = array();
  foreach($pinyin_to_utf8 as $pinyin => $utf8_array) {
    printf("%s,",$pinyin);
    flush();

    $entry = array();
    // generate char array
    $chars = array();
    foreach($utf8_array as $utf8) {
      $tmp = array();
      $tmp['ch'] = $utf8;
      if(key_exists($utf8,$word_counts)) $tmp['count'] = $word_counts[$utf8];
      else $tmp['count'] = 0;
      $chars[] = $tmp;
    }
    usort($chars,"sort_compare1");
    $entry['chars'] = $chars;

    $relations = array();
    // generate relation array
    foreach($word_counts as $word => $count) {
      if(utf8strlen($word) == 1) continue; // skip one word
      // get the first character of the word
      $utf8 = substr($word,0,utf8len($word[0]));
      $first_pinyin = $utf8_to_pinyin[$utf8];

      $skip = true;
      for($i=0;$i<count($first_pinyin);$i++) { 
	if($first_pinyin[$i] === $pinyin) {
	  $skip = false;
	  break;
	}
      }

      // the pinyin does not match
      if($skip) continue;

      // this is a keyword
      $tmp = array();
      $tmp['count'] = $count;
      $tmp['word'] = $word;
      // generate pinyin
      for($first=true,$pinyins='',$i=0;$i<strlen($word);$first=false) {
	$len = utf8len($word[$i]);
	$utf8 = substr($word,$i,$len);
	$word_pinyin = $utf8_to_pinyin[$utf8];
	$word_pinyin = $word_pinyin[0]; // FIXME: get the first one may be wrong...
	if(!$first) {
	  $pinyins .= $word_pinyin;
	  $pinyins .= ' ';
	}
	$i += $len;
      }
      $pinyins = rtrim($pinyins,' ');

      if(key_exists($pinyins,$relations)) {
	$tmp2 = $relations[$pinyins];
	$tmp2[] = $tmp;
	$relations[$pinyins] = $tmp2;
      }
      else {
	$relations[$pinyins] = array($tmp);
      }
    }
    // sort the array inside relations
    foreach($relations as $relation) {
      usort($relation,"sort_compare2");
    }
    ksort($relations);
    $entry['relations'] = $relations;
    //
    $table[$pinyin] = $entry;
  }
  printf("...Done\n");
  return $table;
}


if(count($argv) < 4) {
  printf("php %s (pinyin_table) (a chinese text in utf8) (output file)\n",$argv[0]);
  return;
}

if(file_exists($CACHE)) {
  printf("Using cache %s...\n",$CACHE);
  $cache = unserialize(file_get_contents($CACHE));
  extract($cache);
}

$new_cache = false;
if(!$pinyin_to_utf8) {
  $pinyin_to_utf8 = generate_pinyin_to_utf8_table($argv[1]);
  $new_cache = true;
}
if(!$utf8_to_pinyin) {
  $utf8_to_pinyin = generate_utf8_to_pinyin_table($pinyin_to_utf8);
  $new_cache = true;
}
if(!$word_counts) {
  $word_counts = generate_word_counts($argv[2],$utf8_to_pinyin,8);
  $new_cache = true;
}
if(!$final_table) {
  $final_table = generate_final_table($pinyin_to_utf8,$utf8_to_pinyin,$word_counts);
  $new_cache = true;
}

if($new_cache) {
  // write cache
  $cache = array();
  $cache['pinyin_to_utf8'] = $pinyin_to_utf8;
  $cache['utf8_to_pinyin'] = $utf8_to_pinyin;
  $cache['word_counts'] = $word_counts;
  $cache['final_table'] = $final_table;
  file_put_contents($CACHE,serialize($cache));
  printf("Written cache %s.\n",$CACHE);
  $cache = array();
}
// ok ...we can output the final table as javascript (or JSON)
printf("Outputing table to %s...",$argv[3]);
flush();

$fp = fopen($argv[3],'w');
fprintf($fp,"table: {\n");
foreach($final_table as $pinyin => $table) {
  fprintf($fp,"  '%s': {\n",$pinyin);

  // print chars first
  $buf = '';
  foreach($table['chars'] as $ch) {
    $buf .= sprintf("0x%x,",readutf8($ch['ch']));
  }
  $buf = rtrim($buf,',');
  fprintf($fp,"    chars: [%s],\n",$buf);
  // print relations
  fprintf($fp,"    relations: {\n");
  $buf2 = '';
  foreach($table['relations'] as $pinyins => $relation) {
    $buf = '';
    foreach($relation as $words) {
      $buf .= sprintf("%s,",js_utf8str($words['word']));
    }
    $buf = rtrim($buf,',');
    $buf2 .= sprintf("      '%s': [%s],\n",$pinyins,$buf);
  }
  $buf2 = substr($buf2,0,strlen($buf2)-2);
  fprintf($fp,"%s\n",$buf2);
  fprintf($fp,"    }\n");
  fprintf($fp,"  },\n");
}

fprintf($fp,"}\n");

fclose($fp);
printf("Done\n");
flush();
//print_r($final_table);
//print_r($pinyin_to_utf8);
//print_r($utf8_to_pinyin);
//print_r($word_counts);

?>
